package com.xiaotu.spider.maoyan.pageprocessor;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.model.MaoYanStarDetailModel;
import com.xiaotu.common.redis.RedisClientTemplate;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.DateUtils;
import com.xiaotu.common.util.GsonUtils;
import com.xiaotu.common.util.HttpUtils;
import com.xiaotu.common.util.KeywordEncode;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.PropertiesUtil;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

/**
 * @类名 MaoYanStarDetailPageProcessor
 * @日期 2018年6月21日
 * @作者 王艳龙
 * @功能 猫眼艺人详情信息爬取
 */
@Component("MaoYanStarDetailPageProcessor")
public class MaoYanStarDetailPageProcessor extends AUrlPageProcessor{

	@Autowired
	private RedisClientTemplate redisClient;
	private String[] worksArrs = null; //主要作品
	private String[] worktimesArr = null; //作品上映时间
	private String[] workmutlidutysArr = null; //作品角色
	private String[] workmultirolesArr = null; //作品饰演角色
	
	@Override
	protected String[] getStartRequestUrls() {
		List<JSONObject> targetList = getStarname();
		 String[] urls = new String[targetList.size()];
	        int idx = 0;
          for (int i = 0; i < targetList.size(); i++)
          {
              urls[idx] = "http://maoyan.com/query?kw=" 
              			+ KeywordEncode.encode(targetList.get(i).getString("name"));
              idx++;
          }
				return urls;
		//return new String[]{"http://maoyan.com/query?kw="+KeywordEncode.encode("王海燕")};
	}
	/*@Override
	protected String[] getStartRequestUrls() {
		//int starSise = 2152500;
		int starSise = 100;
		//添加需要爬取的艺人信息地址
		String[] url =new String[starSise];
		for (int i = 1; i <= starSise; i++) {
			url[i-1]= "http://maoyan.com/films/celebrity/"+i;
		}
		return url;
		//int starSise = 10;
		//return new String[]{"http://maoyan.com/films/celebrity/3"};
	}*/
	@Override
	public String getPageDataType(Page page)
	{
		
		if (page.getUrl().toString().contains("http://maoyan.com/query?kw"))//搜索页面
			return Constants.PageDataType.SEARCH_LIST;
		if (page.getUrl().toString().contains("http://maoyan.com/films"))//列表页面
			return Constants.PageDataType.STAR_RECORD;
		return Constants.PageDataType.URL;//json格式的列表数据
	}
	
    @Override
    public void process(Page page)
    {
    	Map<String, Object> tvMap = page.getRequest().getExtras();
        switch (this.getPageDataType(page))
        {
            case Constants.PageDataType.SEARCH_LIST:
                this.listProcess(page,tvMap);
                break;
            case Constants.PageDataType.STAR_RECORD:
                this.recordProcess(page);
                break;
        }
    }

    
    
    private void listProcess(Page page,Map<String, Object> tvMap){
    	String link = page.getHtml().xpath("//ul[@class='celebrity-list']//li[1]"
    			+ "//div[@class='item-poster']//a/@href").get();
    	if(StringUtil.isNotEmpty(link)){
    		PageUtils.addTargetRequest(page,link,tvMap);
    	}
    }
    
    
    private void recordProcess(Page page){
		String[] aliasArr =null; //别名
		String[] representativeworkArr = null; //代表作
		String[] occupationArr = null; //职业
		String birthday= null,born= null,country = null,sex= null,height= null,
				bloodtype= null,nation= null,school= null,constellation= null,fansname= null,brokeragefirm= null;
		String urls = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, page.getUrl().toString());
		//获取艺人名称
		String starname = page.getHtml().xpath("//div[@class='shortInfo']//p[@class='china-name cele-name']/text()").get();
		if(StringUtil.isEmpty(starname))
			throw new SpiderException(SpiderExceptionCode.DataInvalideException);
		//获取该一人所有代表作
		List<String> representativeworkList =page.getHtml().xpath("//div[@class='masterpiece']//ul[@class='master-list']//li").all();
		representativeworkArr = new String[representativeworkList.size()];
		for (int i=0; i<representativeworkList.size();i++) {
			Html workHtml = new Html(representativeworkList.get(i));
			String representativework = workHtml.xpath("//li[@class='master-item']//a//div[@class='overlay']//p[@class='movie-name']/text()").get();
			representativeworkArr[i]=representativework;
		}
		//获取艺人简介
		String introduction = page.getHtml().xpath("//div[@class='introduce']//div[@class='folding-content']"
				+ "//p[@class='cele-desc']/text()").get();
		//获取艺人左边详情信息
		List<String> leftDtDetailList = page.getHtml().xpath("//div[@class='introduce']"
				+ "//div[@class='folding-content']//div[@class='celeInfo-list']//dl[@class='dl-left']//dt").all();
		List<String> leftDdDetailList = page.getHtml().xpath("//div[@class='introduce']"
				+ "//div[@class='folding-content']//div[@class='celeInfo-list']//dl[@class='dl-left']//dd").all();
		if(leftDtDetailList.size() != leftDdDetailList.size())
			throw new SpiderException(SpiderExceptionCode.DataInvalideException);
		
		for (int i=0;i<leftDtDetailList.size();i++) {
			Html htmlLefdttStr = new Html(leftDtDetailList.get(i));
			Html htmlLefddtStr = new Html(leftDdDetailList.get(i));
			String htmlLeftdtStrs =htmlLefdttStr.xpath("//dt/text()").get();
			htmlLeftdtStrs=htmlLeftdtStrs.replace(" ", "");
			String htmlLeftddStrs = htmlLefddtStr.xpath("//dd/text()").get();
			htmlLeftddStrs=htmlLeftddStrs.trim();
			if(htmlLeftdtStrs.contains("别名")){
				String[] aliasArrRep = htmlLeftddStrs.split(",");
				aliasArr = new String[aliasArrRep.length];
				for (int aliast=0;aliast<aliasArrRep.length;aliast++) {
					aliasArr[aliast] = aliasArrRep[aliast];
				}
					continue;
			}
			if(htmlLeftdtStrs.contains("出生日期")){
				birthday = htmlLeftddStrs;
				continue;
			}
			if(htmlLeftdtStrs.contains("出生地")){
				born = htmlLeftddStrs;
				continue;
			}
			if(htmlLeftdtStrs.contains("身份")){
				htmlLeftddStrs=htmlLeftddStrs.replace(" ", "").replace(" ", "");
				occupationArr = htmlLeftddStrs.split("\\|");
					continue;			
			}
			if(htmlLeftdtStrs.contains("性别")){
				sex = htmlLeftddStrs;
				continue;
			}
			if(htmlLeftdtStrs.contains("国籍")){
				country =htmlLeftddStrs;
				continue;			
						}
			if(htmlLeftdtStrs.contains("民族")){
				nation = htmlLeftddStrs;
				continue;
			}
		}
		
		//获取艺人右边详情信息
		List<String> rightDtDetailList = page.getHtml().xpath("//div[@class='introduce']"
				+ "//div[@class='folding-content']//div[@class='celeInfo-list']//dl[@class='dl-right']//dt").all();
		List<String> rightDdDetailList = page.getHtml().xpath("//div[@class='introduce']"
				+ "//div[@class='folding-content']//div[@class='celeInfo-list']//dl[@class='dl-right']//dd").all();
		if(rightDtDetailList.size() != rightDdDetailList.size())
			throw new SpiderException(SpiderExceptionCode.DataInvalideException);
		
		for (int i=0;i<rightDtDetailList.size();i++) {
			Html htmlRightdtStr = new Html(rightDtDetailList.get(i));
			Html htmlRightddStr = new Html(rightDdDetailList.get(i));
			String htmlRightdtStrs =htmlRightdtStr.xpath("//dt/text()").get();
			htmlRightdtStrs=htmlRightdtStrs.replace(" ", "");
			String htmlRightddStrs = htmlRightddStr.xpath("//dd/text()").get();
			htmlRightddStrs=htmlRightddStrs.trim();
			if(htmlRightdtStrs.contains("血型")){
				bloodtype = htmlRightddStrs;
				continue;
			}
			if(htmlRightdtStrs.contains("身高")){
				height = htmlRightddStrs;
				continue;
			}
			if(htmlRightdtStrs.contains("星座")){
				constellation = htmlRightddStrs;
				continue;			
			    		}
			if(htmlRightdtStrs.contains("毕业学校")){
				school = htmlRightddStrs;
				continue;
			}
			if(htmlRightdtStrs.contains("粉丝名称")){
				fansname =htmlRightddStrs;
				continue;			
						}
			if(htmlRightdtStrs.contains("经纪公司")){
				brokeragefirm = htmlRightddStrs;
				continue;
			}
		}
	
	
	//设置爬取艺人作品请求头信息参数
	Map<String, String> headers= new HashMap<String, String>();
	headers.put("Host", "maoyan.com");
	headers.put("Connection", "keep-alive");
	headers.put("Accept", "*/*");
	headers.put("X-Requested-With", "XMLHttpRequest");
	headers.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36");
	headers.put("Accept-Encoding", "gzip, deflate");
	headers.put("Accept-Language", "zh-CN,zh;q=0.9");
	headers.put("Cookie", "_lxsdk_cuid=163ca2cf15ec8-07665d03c8f747-44410a2e-1fa400-163ca2cf15ec8; uuid=1A6E888B4A4B29B16FBA1299108DBE9C1DBAD6193DEE5E37879E485215DB11A8; _csrf=c1a85ef528302cb70e3d515e91fa1f2e58d57d4e2ab9e8d3d1a86fbde1e6b5c3; _lxsdk=1A6E888B4A4B29B16FBA1299108DBE9C1DBAD6193DEE5E37879E485215DB11A8; lt=LqI6cY8Q9bqbkfTjxU8FDq9xxKIAAAAAAwYAAEwVZI_284sTJbBZmKSgx9BRXpp53eZ5AXlPkid9H-Wvhb1rcHI8U00B26Xwgm91sg; lt.sig=9-0U0jm9Aa0bPIdnmVmyGlqxkh4; __mta=209949994.1529544911588.1529632536986.1529633131796.143; _lxsdk_s=16425340a96-e0e-db5-c07%7C%7C9");
	
	StringBuffer worksArrsBuf =new StringBuffer(); //存放艺人作品字符
	StringBuffer worktimesArrBuf =new StringBuffer(); //存放艺人主要作品时间字符
	StringBuffer workmutlidutysArrBuf =new StringBuffer(); //存放艺人作品角色字符
	StringBuffer workmultirolesArrBuf =new StringBuffer(); //存放艺人作品饰演角色字符
	//发送请求获取艺人作品JSON数据 规则offset参数需要0,3,6,9,12、、、、、、
	int offset = 0;
	for(int i =0; i<=100; i++){
		offset = i*3;
		String dataJsons = HttpUtils.httpGetJSONP("http://maoyan.com/films/celebrity/ajax/works/"+urls+"?offset="+offset,headers);
		List<String> resDateAll = RegexUtils.regexSearchAll(RegexUtils.REGEX_INTEGER+"年", dataJsons);
		if(StringUtil.isEmpty(resDateAll)){
			break;
		}
		Set<String> setDate  =new HashSet<String>(resDateAll);
		JSONObject jsonOneList = JSONObject.fromObject(dataJsons);
		List<JSONObject> jsonOne = (List<JSONObject>) jsonOneList.get("data");
		if(jsonOne.size()==0){
			break;
		}
		//解析获取data数据中需要的属性值
		for (String dates : setDate) {
			for(int y=0;y<jsonOne.size();y++){
				List<JSONObject> jsonThree = (List<JSONObject>) jsonOne.get(y).get(dates);
				if(StringUtil.isEmpty(jsonThree)){
					continue;
				}
				//作品、作品上映时间、作品角色、作品饰演角色 以“|”拼接字符
				for (JSONObject dataThree : jsonThree) {
					String names = dataThree.get("name").toString();
					if(StringUtil.isEmpty(names)){
						names = null;
					}
					worksArrsBuf.append(names==null?names:names.replace(",", "/")).append("|");
					String titles = dataThree.get("title").toString();
					if(StringUtil.isEmpty(titles)){
						titles = null;
					}
					worktimesArrBuf.append(titles==null?titles:titles.toString().replace(",", "/")).append("|");
					String mutlidutys = dataThree.get("mutlidutys").toString();
					if(StringUtil.isEmpty(mutlidutys)){
						mutlidutys = null;
					}
					workmutlidutysArrBuf.append(mutlidutys==null?mutlidutys:mutlidutys.replace(",", "/")).append("|");
					String multiroles =dataThree.get("multiroles").toString();
					if(StringUtil.isEmpty(multiroles)){
						multiroles=null;
					}
					workmultirolesArrBuf.append(multiroles==null?multiroles:multiroles.replace(",", "/")).append("|");
				}
					break;
			}
		}
	}
	
		//分割作品字符  并且保存到作品数组中
		if(worksArrsBuf.length()>0){
			String worksArrsBufs = worksArrsBuf.toString().substring(0, worksArrsBuf.toString().length()-1);
			String[] worksArrsStr = worksArrsBufs.toString().split("\\|");
			worksArrs =new String[worksArrsStr.length];
			for (int works=0;works<worksArrsStr.length;works++) {
				worksArrs[works] =worksArrsStr[works];
			}
		}
		//分割作品上映时间字符  并且保存到作品数组中
		if(worktimesArrBuf.length()>0){
			String worktimesArrBufs = worktimesArrBuf.toString().substring(0, worktimesArrBuf.toString().length()-1);
			String[] worktimesArrStr = worktimesArrBufs.toString().split("\\|");
			worktimesArr =new String[worktimesArrStr.length];
			for (int worktimes=0;worktimes<worktimesArrStr.length;worktimes++) {
				worktimesArr[worktimes] =worktimesArrStr[worktimes];
			}
		}
		//分割作品角色字符  并且保存到作品数组中
		if(workmutlidutysArrBuf.length()>0){
			String workmutlidutysArrBufs = workmutlidutysArrBuf.toString().substring(0, workmutlidutysArrBuf.toString().length()-1);
			String[] workmutlidutysArrStr = workmutlidutysArrBufs.toString().split("\\|");
			workmutlidutysArr =new String[workmutlidutysArrStr.length];
			for (int workmutlidutys=0;workmutlidutys<workmutlidutysArrStr.length;workmutlidutys++) {
				workmutlidutysArr[workmutlidutys] =workmutlidutysArrStr[workmutlidutys];
			}
		}
		//分割作品饰演角色字符  并且保存到作品数组中
		if(workmultirolesArrBuf.length()>0){
			String workmultirolesArrBufs = workmultirolesArrBuf.toString().substring(0, workmultirolesArrBuf.toString().length()-1);
			String[] workmultirolesArrStr = workmultirolesArrBufs.toString().split("\\|");
			workmultirolesArr =new String[workmultirolesArrStr.length];
			for (int workmultiroles=0;workmultiroles<workmultirolesArrStr.length;workmultiroles++) {
				workmultirolesArr[workmultiroles] =workmultirolesArrStr[workmultiroles];
			}
}
	
		//缓存爬取过得未播剧名称
    	redisClient.hset(Constants.DataRedisKey.MAOYAN_STAR_DETAIL_ALL.toString(), starname+"_"+urls, DateUtils
                .fomartDateToString(new Date(), DateUtils.SECONDS_FORMAT));
		
		//添加对象属性到缓存中
		page.putField(MaoYanStarDetailModel.class.getName(), new MaoYanStarDetailModel(
				starname, aliasArr, occupationArr, country, sex, height, bloodtype, 
				nation, school, born, birthday, constellation, introduction,fansname,
				brokeragefirm, worksArrs, worktimesArr, workmutlidutysArr, workmultirolesArr,
				representativeworkArr));
    }
    
    public List<JSONObject> getStarname(){
   	 	List<JSONObject> jsonList = new ArrayList<JSONObject>();
   	 	 //获取缓存所有的艺人
		 Map<String, String> targetMap = redisClient.hgetAll(Constants.DataRedisKey.Star_Info.toString());
	     Set<String> set =targetMap.keySet();
	     //获取已经爬取过得缓存艺人
	     Map<String, String> temp = redisClient.hgetAll(Constants.DataRedisKey.MAOYAN_STAR_DETAIL_SNAP.toString());
	     Object starname = null;
	     int length = 30;
	     //设置每次爬取30 艺人  所有艺人缓存与如果已经爬取过得艺人缓存 比较
	     for(String str:set){
	    	 if(temp.containsKey(str)){
	    		 continue;
	    	 }
	    	 starname=targetMap.get(str);
	    	 //缓存爬取过得艺人
	    	 redisClient.hset(Constants.DataRedisKey.MAOYAN_STAR_DETAIL_SNAP.toString(), str, starname.toString());
	    	 //设置爬取过得艺人失效时间
	    	 int timeout= Integer.parseInt(PropertiesUtil.getProperty(Constants.SpiderConfigKey.STAR_INFO_SNAP_TIMEOUT));
	    	 //如果没有设置失效时间   设置获取的失效时间
	    	 if (redisClient.ttl(Constants.DataRedisKey.MAOYAN_STAR_DETAIL_SNAP.toString()) == -1)
	 			redisClient.expire(Constants.DataRedisKey.MAOYAN_STAR_DETAIL_SNAP.toString(), timeout);
	         JSONObject jsonObj = GsonUtils.fromJson(starname.toString(),
                    JSONObject.class);
            jsonList.add(0, jsonObj);
	         length--;
	         if(length==0){
	             break;
	         }
	     }
	     	return jsonList;
   }
    
    
    @Override
    protected String getSpiderType()
    {
        return "MaoYanStarDetail";
    }
}
